1 Setup

Libraries

suppressPackageStartupMessages({
  library(here)
  library(ggplot2)
  library(gplots)
  library(LSD)
  library(pander)
  library(pracma)
  library(RColorBrewer)
  library(viridis)
})

Palette

hpal <- colorRampPalette(c("blue","white","red"))(100)
cols <- brewer.pal(12,"Paired")
mar <- par("mar")

2 Data

AUCs <- sapply(dir(here("data/seidr/roc"),
                   pattern=".*_roc.tsv$",full.names=TRUE),
               function(f){
                 
                 algo <- sub("[^_]+_","",sub("_roc\\.tsv","",f))
                 
                 dat <- read.delim(f,header=FALSE,skip=1,col.names = c("TP","FP","PR"))
                 
                 head <- scan(f,nmax = 3,what = "character",sep="\t")
                 
                 message(sprintf("Found %s GS edges out of %s edges",head[2],sum(as.integer(head[2:3]))))
                 
                 auc <- round(trapz(dat[,2],dat[,1]),digits=3)
                 
                 plot(dat[,2],dat[,1],type="l",main=sprintf("%s (AUC = %s)",algo,auc),
                      xlab="False Positive Rate",ylab="True Positive Rate",
                      sub=sprintf("%s Gold Standard edges out of %s edges\n%s",head[2],sum(as.integer(head[2:3])),
                                  basename(f)))
                 
                 abline(0,1,lty=2)
                 
                 message(sprintf("The AUC is %s",auc))
                 
                 return(c(sub("_.*","",basename(f)),algo,auc))
               })
## Found 79 GS edges out of 30856 edges
## The AUC is 0.454
## Found 215792 GS edges out of 240236315 edges

## The AUC is 0.567
## Found 164057 GS edges out of 166252177 edges

## The AUC is 0.619
## Found 146353 GS edges out of 201569549 edges

## The AUC is 0.601
## Found 222561 GS edges out of 464865786 edges

## The AUC is 0.763
## Found 3179 GS edges out of 48894954 edges

## The AUC is 0.236
## Found 222561 GS edges out of 464865786 edges

## The AUC is 0.763
## Found 2541 GS edges out of 26431075 edges

## The AUC is 0.847
## Found 222561 GS edges out of 464865786 edges

## The AUC is 0.66
## Found 222561 GS edges out of 464865786 edges

## The AUC is 0.788
## Found 222559 GS edges out of 464860283 edges

## The AUC is 0.614
## Found 222561 GS edges out of 464863253 edges

## The AUC is 0.721
## Found 33621 GS edges out of 21936947 edges

## The AUC is 0.586
## Found 5 GS edges out of 3737 edges

## The AUC is 0.631
## Found 7949 GS edges out of 1386334 edges

## The AUC is 0.47
## Found 7953 GS edges out of 1324170 edges

## The AUC is 0.812
## Found 7903 GS edges out of 1334447 edges

## The AUC is 0.678
## Found 7965 GS edges out of 1390175 edges

## The AUC is 0.739
## Found 125 GS edges out of 641301 edges

## The AUC is 0.523
## Found 7965 GS edges out of 1390175 edges

## The AUC is 0.529
## Found 994 GS edges out of 166099 edges

## The AUC is 0.788
## Found 7965 GS edges out of 1390175 edges

## The AUC is 0.674
## Found 7965 GS edges out of 1390175 edges

## The AUC is 0.73
## Found 7965 GS edges out of 1390155 edges

## The AUC is 0.498
## Found 7965 GS edges out of 1390175 edges

## The AUC is 0.827
## Found 7461 GS edges out of 1338013 edges

## The AUC is 0.514
## Found 1 GS edges out of 1757 edges

## The AUC is 0.06
## Found 854 GS edges out of 63605 edges

## The AUC is 0.473
## Found 854 GS edges out of 62361 edges

## The AUC is 0.766
## Found 854 GS edges out of 63177 edges

## The AUC is 0.592
## Found 855 GS edges out of 63656 edges

## The AUC is 0.757
## Found 9 GS edges out of 16347 edges

## The AUC is 0.431
## Found 855 GS edges out of 63656 edges

## The AUC is 0.667
## Found 495 GS edges out of 27719 edges

## The AUC is 0.779
## Found 855 GS edges out of 63656 edges

## The AUC is 0.545
## Found 855 GS edges out of 63656 edges

## The AUC is 0.736
## Found 855 GS edges out of 63654 edges

## The AUC is 0.344
## Found 855 GS edges out of 63656 edges

## The AUC is 0.758
## Found 851 GS edges out of 63086 edges

## The AUC is 0.604
## Found 1 GS edges out of 2031 edges

## The AUC is 0.054
## Found 1656 GS edges out of 129175 edges

## The AUC is 0.504
## Found 1658 GS edges out of 126172 edges

## The AUC is 0.789
## Found 1658 GS edges out of 128011 edges

## The AUC is 0.611
## Found 1659 GS edges out of 129323 edges

## The AUC is 0.74
## Found 20 GS edges out of 40283 edges

## The AUC is 0.436
## Found 1659 GS edges out of 129323 edges

## The AUC is 0.653
## Found 622 GS edges out of 45838 edges

## The AUC is 0.783
## Found 1659 GS edges out of 129323 edges

## The AUC is 0.548
## Found 1659 GS edges out of 129323 edges

## The AUC is 0.746
## Found 1659 GS edges out of 129320 edges

## The AUC is 0.351
## Found 1659 GS edges out of 129323 edges

## The AUC is 0.79
## Found 1639 GS edges out of 127703 edges

## The AUC is 0.575
## Found 1 GS edges out of 2261 edges

## The AUC is 0.049
## Found 2439 GS edges out of 206779 edges

## The AUC is 0.507
## Found 2443 GS edges out of 201466 edges

## The AUC is 0.794
## Found 2442 GS edges out of 204315 edges

## The AUC is 0.628
## Found 2444 GS edges out of 207073 edges

## The AUC is 0.731
## Found 34 GS edges out of 71418 edges

## The AUC is 0.532
## Found 2444 GS edges out of 207073 edges

## The AUC is 0.63
## Found 712 GS edges out of 61958 edges

## The AUC is 0.778
## Found 2444 GS edges out of 207073 edges

## The AUC is 0.569
## Found 2444 GS edges out of 207073 edges

## The AUC is 0.742
## Found 2444 GS edges out of 207070 edges

## The AUC is 0.375
## Found 2444 GS edges out of 207073 edges

## The AUC is 0.801
## Found 2406 GS edges out of 204022 edges

## The AUC is 0.55
## Found 2 GS edges out of 2443 edges

## The AUC is 0.492
## Found 3206 GS edges out of 302715 edges

## The AUC is 0.505
## Found 3209 GS edges out of 294143 edges

## The AUC is 0.804
## Found 3206 GS edges out of 298155 edges

## The AUC is 0.641
## Found 3211 GS edges out of 303241 edges

## The AUC is 0.732
## Found 44 GS edges out of 112782 edges

## The AUC is 0.588
## Found 3211 GS edges out of 303241 edges

## The AUC is 0.61
## Found 764 GS edges out of 77665 edges

## The AUC is 0.782
## Found 3211 GS edges out of 303241 edges

## The AUC is 0.59
## Found 3211 GS edges out of 303241 edges

## The AUC is 0.741
## Found 3211 GS edges out of 303237 edges

## The AUC is 0.395
## Found 3211 GS edges out of 303241 edges

## The AUC is 0.809
## Found 3155 GS edges out of 297960 edges

## The AUC is 0.536
## Found 4 GS edges out of 2656 edges

## The AUC is 0.69
## Found 4035 GS edges out of 423679 edges

## The AUC is 0.501
## Found 4036 GS edges out of 410422 edges

## The AUC is 0.809
## Found 4033 GS edges out of 415791 edges

## The AUC is 0.655
## Found 4041 GS edges out of 424495 edges

## The AUC is 0.734
## Found 58 GS edges out of 167359 edges

## The AUC is 0.589
## Found 4041 GS edges out of 424495 edges

## The AUC is 0.593
## Found 806 GS edges out of 93594 edges

## The AUC is 0.785
## Found 4041 GS edges out of 424495 edges

## The AUC is 0.61
## Found 4041 GS edges out of 424495 edges

## The AUC is 0.74
## Found 4041 GS edges out of 424487 edges

## The AUC is 0.417
## Found 4041 GS edges out of 424495 edges

## The AUC is 0.817
## Found 3966 GS edges out of 415989 edges

## The AUC is 0.522
## Found 4 GS edges out of 2828 edges

## The AUC is 0.674
## Found 4826 GS edges out of 562321 edges

## The AUC is 0.496
## Found 4826 GS edges out of 543347 edges

## The AUC is 0.812
## Found 4817 GS edges out of 549872 edges

## The AUC is 0.66
## Found 4833 GS edges out of 563494 edges

## The AUC is 0.735
## Found 75 GS edges out of 232132 edges

## The AUC is 0.55
## Found 4833 GS edges out of 563494 edges

## The AUC is 0.578
## Found 852 GS edges out of 108533 edges

## The AUC is 0.787
## Found 4833 GS edges out of 563494 edges

## The AUC is 0.625
## Found 4833 GS edges out of 563494 edges

## The AUC is 0.739
## Found 4833 GS edges out of 563483 edges

## The AUC is 0.435
## Found 4833 GS edges out of 563494 edges

## The AUC is 0.823
## Found 4713 GS edges out of 550807 edges

## The AUC is 0.512
## Found 4 GS edges out of 3019 edges

## The AUC is 0.656
## Found 5574 GS edges out of 705886 edges

## The AUC is 0.49
## Found 5575 GS edges out of 680493 edges

## The AUC is 0.813
## Found 5559 GS edges out of 688064 edges

## The AUC is 0.665
## Found 5582 GS edges out of 707511 edges

## The AUC is 0.735
## Found 84 GS edges out of 301117 edges

## The AUC is 0.553
## Found 5582 GS edges out of 707511 edges

## The AUC is 0.566
## Found 892 GS edges out of 121657 edges

## The AUC is 0.788
## Found 5582 GS edges out of 707511 edges

## The AUC is 0.638
## Found 5582 GS edges out of 707511 edges

## The AUC is 0.738
## Found 5582 GS edges out of 707498 edges

## The AUC is 0.45
## Found 5582 GS edges out of 707511 edges

## The AUC is 0.824
## Found 5410 GS edges out of 689674 edges

## The AUC is 0.506
## Found 5 GS edges out of 3239 edges

## The AUC is 0.672
## Found 6326 GS edges out of 890325 edges

## The AUC is 0.484
## Found 6325 GS edges out of 855698 edges

## The AUC is 0.813
## Found 6304 GS edges out of 864523 edges

## The AUC is 0.671
## Found 6335 GS edges out of 892514 edges

## The AUC is 0.738
## Found 93 GS edges out of 391311 edges

## The AUC is 0.553
## Found 6335 GS edges out of 892514 edges

## The AUC is 0.553
## Found 930 GS edges out of 135963 edges

## The AUC is 0.787
## Found 6335 GS edges out of 892514 edges

## The AUC is 0.651
## Found 6335 GS edges out of 892514 edges

## The AUC is 0.735
## Found 6335 GS edges out of 892497 edges

## The AUC is 0.467
## Found 6335 GS edges out of 892514 edges

## The AUC is 0.827
## Found 6083 GS edges out of 866956 edges

## The AUC is 0.506
## Found 5 GS edges out of 3493 edges

## The AUC is 0.651
## Found 7132 GS edges out of 1127869 edges

## The AUC is 0.476
## Found 7133 GS edges out of 1080524 edges

## The AUC is 0.812
## Found 7100 GS edges out of 1090450 edges

## The AUC is 0.675
## Found 7144 GS edges out of 1130829 edges

## The AUC is 0.74
## Found 110 GS edges out of 510463 edges

## The AUC is 0.543
## Found 7144 GS edges out of 1130829 edges

## The AUC is 0.539
## Found 968 GS edges out of 151492 edges

## The AUC is 0.786
## Found 7144 GS edges out of 1130829 edges

## The AUC is 0.664
## Found 7144 GS edges out of 1130829 edges

## The AUC is 0.733
## Found 7144 GS edges out of 1130811 edges

## The AUC is 0.485
## Found 7144 GS edges out of 1130829 edges

## The AUC is 0.829
## Found 6778 GS edges out of 1093491 edges

## The AUC is 0.509
## Found 19 GS edges out of 11218 edges

## The AUC is 0.401
## Found 10324 GS edges out of 1759138 edges

## The AUC is 0.259
## Found 10315 GS edges out of 1636546 edges

## The AUC is 0.81
## Found 10207 GS edges out of 1635291 edges

## The AUC is 0.639
## Found 10330 GS edges out of 1760498 edges

## The AUC is 0.665
## Found 236 GS edges out of 884754 edges

## The AUC is 0.576
## Found 10330 GS edges out of 1760498 edges

## The AUC is 0.293
## Found 1070 GS edges out of 176869 edges

## The AUC is 0.757
## Found 10330 GS edges out of 1760498 edges

## The AUC is 0.702
## Found 10330 GS edges out of 1760498 edges

## The AUC is 0.662
## Found 10330 GS edges out of 1760482 edges

## The AUC is 0.618
## Found 10330 GS edges out of 1760498 edges

## The AUC is 0.841
## Found 8510 GS edges out of 1529623 edges

## The AUC is 0.551

tab <- split.data.frame(t(AUCs[2:3,]),AUCs[1,])

stopifnot(nrow(unique(t(sapply(tab,function(ta){ta[,1]})))) == 1)

res <- sapply(tab,function(ta){as.numeric(ta[,2])})
rownames(res) <- tab[[1]][,1]
res <- res[,c(3:(ncol(res)-1),2,1,ncol(res))]

Re-order the data.

rocord <- c("backbone-1-percent", "backbone-2-percent", "backbone-3-percent",
            "backbone-4-percent", "backbone-5-percent", "backbone-6-percent", 
            "backbone-7-percent", "backbone-8-percent", "backbone-9-percent", 
            "backbone-10-percent", "aggregated","hard-threshold-dot3")

Create a heatmap of the AUC values for each network and for each algorithm.

heatmap.2(res,
          trace="none",
          margins = c(10.1,7.1),
          col = hpal)

heatmap.2(as.matrix(res[,rocord]), 
          Colv = FALSE,
          dendrogram = "row",
          trace="none",
          margins = c(10.1,7.1),
          col = hpal)

Obtain a line-plot from this heatmap.

par(mar=c(10.1,3.1,3.1,0.1))
linesplot(res[,rocord],cols = colorRampPalette(c("blue","red"))(20),
          addboxes = TRUE,las=2,main="AUC distribution")

par(mar=mar)

par(mar=c(10.1,4.1,3.1,2.1))
plot(0,0,ylim=c(0,1),
     xlim=c(1,ncol(res)),type="n",
     xaxt="n",xlab="",ylab="AUC",main="AUC evolution")
axis(1,1:ncol(res),las=2,labels=colnames(res))
sapply(1:nrow(res),function(i){lines(res[i,],col=cols[i],lwd=2,lty=2)})
## [[1]]
## NULL
## 
## [[2]]
## NULL
## 
## [[3]]
## NULL
## 
## [[4]]
## NULL
## 
## [[5]]
## NULL
## 
## [[6]]
## NULL
## 
## [[7]]
## NULL
## 
## [[8]]
## NULL
## 
## [[9]]
## NULL
## 
## [[10]]
## NULL
## 
## [[11]]
## NULL
## 
## [[12]]
## NULL
## 
## [[13]]
## NULL
legend("bottomright",legend = rownames(res),col=cols[1:nrow(res)],lty=1,lwd=2)

par(mar=mar)

pander(res)
Table continues below
  backbone-1-percent backbone-2-percent backbone-3-percent
ARACNE 0.06 0.054 0.049
CLR 0.473 0.504 0.507
ELNET 0.766 0.789 0.794
GENIE3 0.592 0.611 0.628
irp 0.757 0.74 0.731
LLR 0.431 0.436 0.532
MI 0.667 0.653 0.63
NARROMI 0.779 0.783 0.778
PCOR 0.545 0.548 0.569
PEARSON 0.736 0.746 0.742
PLSNET 0.344 0.351 0.375
SPEARMAN 0.758 0.79 0.801
TIGRESS 0.604 0.575 0.55
Table continues below
  backbone-4-percent backbone-5-percent backbone-6-percent
ARACNE 0.492 0.69 0.674
CLR 0.505 0.501 0.496
ELNET 0.804 0.809 0.812
GENIE3 0.641 0.655 0.66
irp 0.732 0.734 0.735
LLR 0.588 0.589 0.55
MI 0.61 0.593 0.578
NARROMI 0.782 0.785 0.787
PCOR 0.59 0.61 0.625
PEARSON 0.741 0.74 0.739
PLSNET 0.395 0.417 0.435
SPEARMAN 0.809 0.817 0.823
TIGRESS 0.536 0.522 0.512
Table continues below
  backbone-7-percent backbone-8-percent backbone-9-percent
ARACNE 0.656 0.672 0.651
CLR 0.49 0.484 0.476
ELNET 0.813 0.813 0.812
GENIE3 0.665 0.671 0.675
irp 0.735 0.738 0.74
LLR 0.553 0.553 0.543
MI 0.566 0.553 0.539
NARROMI 0.788 0.787 0.786
PCOR 0.638 0.651 0.664
PEARSON 0.738 0.735 0.733
PLSNET 0.45 0.467 0.485
SPEARMAN 0.824 0.827 0.829
TIGRESS 0.506 0.506 0.509
  backbone-10-percent aggregated hard-threshold-dot3
ARACNE 0.631 0.454 0.401
CLR 0.47 0.567 0.259
ELNET 0.812 0.619 0.81
GENIE3 0.678 0.601 0.639
irp 0.739 0.763 0.665
LLR 0.523 0.236 0.576
MI 0.529 0.763 0.293
NARROMI 0.788 0.847 0.757
PCOR 0.674 0.66 0.702
PEARSON 0.73 0.788 0.662
PLSNET 0.498 0.614 0.618
SPEARMAN 0.827 0.721 0.841
TIGRESS 0.514 0.586 0.551 
write.csv(res[,rocord],file=here("data/seidr/roc/AUCs.csv"),quote=FALSE)

3 Session Info

## R version 3.6.1 (2019-07-05)
## Platform: x86_64-pc-linux-gnu (64-bit)
## Running under: Ubuntu 18.04.3 LTS
## 
## Matrix products: default
## BLAS/LAPACK: /usr/lib/x86_64-linux-gnu/libopenblasp-r0.2.20.so
## 
## locale:
##  [1] LC_CTYPE=en_US.UTF-8       LC_NUMERIC=C              
##  [3] LC_TIME=en_US.UTF-8        LC_COLLATE=en_US.UTF-8    
##  [5] LC_MONETARY=en_US.UTF-8    LC_MESSAGES=en_US.UTF-8   
##  [7] LC_PAPER=en_US.UTF-8       LC_NAME=C                 
##  [9] LC_ADDRESS=C               LC_TELEPHONE=C            
## [11] LC_MEASUREMENT=en_US.UTF-8 LC_IDENTIFICATION=C       
## 
## attached base packages:
## [1] stats     graphics  grDevices utils     datasets  methods   base     
## 
## other attached packages:
## [1] viridis_0.5.1      viridisLite_0.3.0  RColorBrewer_1.1-2 pracma_2.2.5      
## [5] pander_0.6.3       LSD_4.0-0          gplots_3.0.1.1     ggplot2_3.2.1     
## [9] here_0.1          
## 
## loaded via a namespace (and not attached):
##  [1] Rcpp_1.0.3         pillar_1.4.2       compiler_3.6.1     highr_0.8         
##  [5] bitops_1.0-6       tools_3.6.1        digest_0.6.23      evaluate_0.14     
##  [9] lifecycle_0.1.0    tibble_2.1.3       gtable_0.3.0       pkgconfig_2.0.3   
## [13] rlang_0.4.2        yaml_2.2.0         xfun_0.11          gridExtra_2.3     
## [17] withr_2.1.2        stringr_1.4.0      dplyr_0.8.3        knitr_1.26        
## [21] gtools_3.8.1       caTools_1.17.1.2   rprojroot_1.3-2    grid_3.6.1        
## [25] tidyselect_0.2.5   glue_1.3.1         R6_2.4.1           rmarkdown_1.18    
## [29] gdata_2.18.0       purrr_0.3.3        magrittr_1.5       backports_1.1.5   
## [33] scales_1.1.0       htmltools_0.4.0    assertthat_0.2.1   colorspace_1.4-1  
## [37] KernSmooth_2.23-16 stringi_1.4.3      lazyeval_0.2.2     munsell_0.5.0     
## [41] crayon_1.3.4